package test.adddata;

import cn.hutool.core.util.ZipUtil;
import cn.hutool.db.Db;
import cn.hutool.db.Entity;
import cn.hutool.db.ds.DSFactory;
import lombok.SneakyThrows;
import utils.SmallTool;

import java.sql.SQLException;
import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;

/**
 * @Author: gcr
 * @Date: 2022-5-23 13:42
 */
public class 查找不存在的pmidxml插入表 {
    @SneakyThrows
    public static void main(String[] args) {

        // 为了防止网络中断直接先把数据插入本地表在更新服务器上的

        // vm options -Xmx68660m -Xms22560m
        String to = "pubmed.xml_content_20200320_noexist_taiyi";
        String taiyiXml = "spider_pubmed_xml_content";
        String from = "pubmed.xml_content_1_3qw";
        Db use = Db.use();
        Db taiyiUse = Db.use(DSFactory.get("taiyi4444"));
        use.execute("truncate table " + to);

        String allDate = "select max(pmid) as pmid  from " + from;
        Entity queryNew = use.query(allDate).get(0);
        SmallTool.printMess("最大的id 是" + queryNew.getStr("pmid"));

        Integer count = queryNew.getInt("pmid");
        int breakSize = 100000;
        int pageSize = count / breakSize + 1;
        SmallTool.printMess("全部页码+++++++++" + pageSize);
        SmallTool.printMess("任务开始+++++++++++++++++++++++++++++++++++++++++++++++++++++");
        for (int i = 1; i <= pageSize; i++) {
//            SmallTool.printMess("当前页码+++++" + i);
            if (i <= 380) {
                continue;
            }
//            SmallTool.printMemory();
            int start = breakSize * (i - 1) + 1;
            int end = i * breakSize;
            SmallTool.printMess("开始id++" + start + "+++结束id++++" + end);
            String paramSql = "select pmid,content from " + from + " where  pmid between  " + start + " and " + end;
            String server = "select pmid from " + taiyiXml + " where  pmid between  " + start + " and " + end;
            List<Entity> querys = use.query(paramSql);
            List<Entity> queryTaiyi = taiyiUse.query(server);
            Set<Integer> mepmid = querys.stream().map(o -> o.getInt("pmid")).collect(Collectors.toSet());
            Set<Integer> serverPmid = queryTaiyi.stream().map(o -> o.getInt("pmid")).collect(Collectors.toSet());
            mepmid.removeAll(serverPmid);

            querys.parallelStream().forEach(entity -> {
                Integer pmid = entity.getInt("pmid");
                if (mepmid.contains(pmid)) {
                    String content = ZipUtil.unGzip(entity.getBytes("content"), "utf-8");
                    try {
                        use.insert(new Entity().setTableName(to).set("pmid", pmid).set("content", ZipUtil.gzip(content, "UTF-8")));
                    } catch (SQLException e) {
                        e.printStackTrace();
                    }
                }
            });
        }

        SmallTool.printMess("不存在xml部分插入泰医服务器开始了++");
        不存在xml部分插入泰医服务器.main(args);
    }
}
